
#include "segment.h"
#include "locale.h"


//--------------------Segment Base class ----------------
Segmentor::Segmentor()
{
	head = new Sentence();
	head->next = 0;
	tail = head;

	document = 0;
}

Segmentor::~Segmentor()
{
	Sentence * next = 0;
	while(head != 0)
	{	
		next = head->next;
		delete head;
		head = next;
	}
	head = 0;
}

void Segmentor::segmentSentences()
{
	for(Sentence* cur = head->next; cur != 0; cur = cur->next)
	{
		if(cur->needSeg)
		{
			segmentSentence(cur);
		}
	}
}

void Segmentor::analysisDocument(wchar_t * wdocument)
{
	this->document = wdocument;

	splitIntoSentences();

	segmentSentences();
	
	generateNewWord();
}


bool Segmentor::analysisFile(const char * path)
{
	FILE * pFile = fopen(path,"r");
	
	if (pFile==NULL) 
	{
		printf("Error opening file.\n");
		return false;
	}
	long size=0;

	//even the file is empty, it'll return 1.
	while(!feof(pFile))
	{
		fgetc(pFile);
		++size;
	}

	rewind(pFile);
	char * buffer = new char[size];
	buffer[size-1] = 0;
	fread(buffer,sizeof(char),size-1,pFile);
	fclose(pFile);

	int wchar_size = CalculateCharToWchar(buffer,-1);
	wchar_t * wbuffer = new wchar_t[wchar_size];
	CharToWchar(buffer,wbuffer,wchar_size-1);
	wbuffer[wchar_size-1] = 0;
	
	delete [] buffer;

	this->analysisDocument(wbuffer);
	
	return true;
}

void Segmentor::addSentence(wchar_t *senten, bool needSeg, int senLen)
{

	Sentence * s = new Sentence();
	s->needSeg = needSeg;
	s->se = senten;
	s->length = senLen;
	s->next = 0;

	tail->next = s;
	tail = s;	
	
	if(head->next == 0)
	{
		head->next = s;
	}
}

bool Segmentor::isChinese(wchar_t* t)
{
	return *t>=0x4e00 && *t<=0x9FA0;
}

bool Segmentor::isAlphabet(wchar_t* t)
{
	return (*t>=65&&*t<=90) || ( *t>= 97 && *t<=122);
}

//----------------------------------------------------------

ChineseSegmentor::ChineseSegmentor():Segmentor()
{

	const char * file = "data/dict.dat";
	const char * rmmFile = "data/reverse_dict.dat";

	dict = new Dictionary(rmmFile,Dictionary::RMM);
	
	patArray = new PatArray<wchar_t>(dict);
	segPatArray = new PatArray<SegmentFragment>(dict);
}

ChineseSegmentor::~ChineseSegmentor()
{
	delete dict;
	delete patArray;
	delete segPatArray;
}

void ChineseSegmentor::splitIntoSentences()
{
	wchar_t *begin=document;
	wchar_t *end=begin;
	
	bool curType=isChinese(end++);
	for(; *end != 0; ++end)
	{
		if(curType !=isChinese(end))
		{
			int size = end-begin;
			wchar_t * newSen;
			if(curType)
			{
				//create a new wchar string being and end with 0.
				newSen = new wchar_t[size+2];
				newSen[0] = newSen[size+1]=0;
				wcsncpy(newSen+1,begin,size);
			}
			else
			{
				newSen = new wchar_t[size+1];
				newSen[size] = 0;
				wcsncpy(newSen,begin,size);
			}
			addSentence(newSen, curType,size); 			
			curType = !curType;
			begin = end;
		}				
	}

	int size = end-begin;
	wchar_t * newSen;
	if(curType)
	{
		//create a new wchar string being and end with 0.
		newSen = new wchar_t[size+2];
		newSen[0] = newSen[size+1]=0;
		wcsncpy(newSen+1,begin,size);
	}
	else
	{
		newSen = new wchar_t[size+1];
		newSen[size] = 0;
		wcsncpy(newSen,begin,size);
	}
	addSentence(newSen, curType,size); 	
}


void ChineseSegmentor::generateNewWord()
{
	for(Sentence* cur = head->next; cur != 0; cur = cur->next)
	{
		if(cur->needSeg)
		{			
			segPatArray->add(cur);
			patArray->add(cur);
		}
	}
	segPatArray->generateNewWord();
	patArray->generateNewWord();
}

void ChineseSegmentor::segmentSentence(Sentence * se)
{
	int base = Trie::BASE_POS;
	int next_base = -1;


	wchar_t *begin = se->se + se->length;
	wchar_t *end = begin;
	wchar_t *last_pos;

	while(*end != 0)
	{
		next_base = dict->forward(base,*end);
		if( dict->is_phrase(next_base))
		{
			last_pos = end-1;   // 
		}
		else
		{
			if( begin==end)
				last_pos = end - 1;
		}

		if( next_base != -1)
		{
			base = next_base;
			--end;
		}else
		{
			se->addSegmentFragment(last_pos+1,begin);

			base = Trie::BASE_POS;
			begin = last_pos;
			end=begin;
		}
	}
}

void ChineseSegmentor::info()
{
//	patArray->printContext();
//	patArray->printPrefixList();
//	patArray->printBoundaryCounterInfo();
//	patArray->printLcpData();
//	patArray->printEntropyInOrder();	

	segPatArray->printEntropyInOrder();
	patArray->printEntropyInOrder();
}



//----------------------------------------------------------
//
//EnglishSegmentor::EnglishSegmentor():Segmentor()
//{
//	patArray =  new PatArray<wchar_t>();
//}
//
//EnglishSegmentor::~EnglishSegmentor()
//{
//	delete patArray;
//}
//
//void EnglishSegmentor::splitIntoSentences()
//{
//	wchar_t *begin=document;
//	wchar_t *end=begin;
//	
//	bool curType;
//	for(; *end != 0; ++end)
//	{
//		if(curType !=isAlphabet(end))
//		{
//			int size = end-begin;
//			wchar_t * newSen;
//			if(curType)
//			{
//				//create a new wchar string being and end with 0.
//				newSen = new wchar_t[size+2];
//				newSen[0] = newSen[size+1]=0;
//				wcsncpy(newSen+1,begin,size);
//			}
//			else
//			{
//				newSen = new wchar_t[size+1];
//				newSen[size] = 0;
//				wcsncpy(newSen,begin,size);
//			}
//			addSentence(newSen, curType,size); 			
//			curType = !curType;
//			begin = end;
//		}				
//	}
//
//	int size = end-begin;
//	wchar_t * newSen;
//	if(curType)
//	{
//		//create a new wchar string being and end with 0.
//		newSen = new wchar_t[size+2];
//		newSen[0] = newSen[size+1]=0;
//		wcsncpy(newSen+1,begin,size);
//	}
//	else
//	{
//		newSen = new wchar_t[size+1];
//		newSen[size] = 0;
//		wcsncpy(newSen,begin,size);
//	}
//	addSentence(newSen, curType,size); 	
//
//}




		